{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 激活函数不同"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "\n",
    "\n",
    "from keras.layers.core import Dense, Flatten\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "\n",
    "from keras import backend as K\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "    x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "\n",
    "net = Conv2D(32, kernel_size=[5,5], strides=[1,1],activation='relu', #使用不同的激活函数relu,tanh,sigmoid\n",
    "                 padding='same',                \n",
    "                input_shape=[28,28,1])(x_image)\n",
    "net = MaxPooling2D(pool_size=[2,2])(net)\n",
    "net = Conv2D(64, kernel_size=[5,5], strides=[1,1],activation='relu',#使用不同的激活函数relu,tanh,sigmoid\n",
    "                padding='same')(net)\n",
    "net = MaxPooling2D(pool_size=[2,2])(net)\n",
    "#net = tf.nn.dropout(keep_prob = 0.5)(net)\n",
    "net = Flatten()(net)\n",
    "net = Dense(1000, activation='relu')(net) #使用不同的激活函数relu,tanh,sigmoid\n",
    "net = Dense(10,activation='softmax')(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.objectives import categorical_crossentropy\n",
    "cross_entropy = tf.reduce_mean(categorical_crossentropy(y_, net))\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "K.set_session(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 1.901965, l2_loss: 3952.243408, total loss: 2.178622\n",
      "0.66\n",
      "step 200, entropy loss: 0.587549, l2_loss: 3953.889404, total loss: 0.864321\n",
      "0.89\n",
      "step 300, entropy loss: 0.474466, l2_loss: 3954.469482, total loss: 0.751279\n",
      "0.91\n",
      "step 400, entropy loss: 0.440821, l2_loss: 3954.558350, total loss: 0.717640\n",
      "0.89\n",
      "step 500, entropy loss: 0.282707, l2_loss: 3954.519531, total loss: 0.559524\n",
      "0.93\n",
      "step 600, entropy loss: 0.191026, l2_loss: 3954.348389, total loss: 0.467831\n",
      "0.94\n",
      "step 700, entropy loss: 0.179702, l2_loss: 3954.169922, total loss: 0.456494\n",
      "0.95\n",
      "step 800, entropy loss: 0.174280, l2_loss: 3953.940186, total loss: 0.451056\n",
      "0.97\n",
      "step 900, entropy loss: 0.283003, l2_loss: 3953.712646, total loss: 0.559763\n",
      "0.96\n",
      "step 1000, entropy loss: 0.278914, l2_loss: 3953.479736, total loss: 0.555657\n",
      "0.94\n",
      "0.9467\n",
      "1000步所用的时间为206.190883s\n",
      "step 1100, entropy loss: 0.176603, l2_loss: 3953.157471, total loss: 0.453324\n",
      "0.96\n",
      "step 1200, entropy loss: 0.152268, l2_loss: 3952.873291, total loss: 0.428969\n",
      "0.96\n",
      "step 1300, entropy loss: 0.155924, l2_loss: 3952.570557, total loss: 0.432604\n",
      "0.97\n",
      "step 1400, entropy loss: 0.148803, l2_loss: 3952.263672, total loss: 0.425462\n",
      "0.97\n",
      "step 1500, entropy loss: 0.084664, l2_loss: 3951.949707, total loss: 0.361301\n",
      "0.98\n",
      "step 1600, entropy loss: 0.124013, l2_loss: 3951.636475, total loss: 0.400627\n",
      "0.98\n",
      "step 1700, entropy loss: 0.159397, l2_loss: 3951.288086, total loss: 0.435987\n",
      "0.96\n",
      "step 1800, entropy loss: 0.111139, l2_loss: 3950.951904, total loss: 0.387706\n",
      "0.99\n",
      "step 1900, entropy loss: 0.093764, l2_loss: 3950.607422, total loss: 0.370306\n",
      "0.99\n",
      "step 2000, entropy loss: 0.096448, l2_loss: 3950.263672, total loss: 0.372966\n",
      "0.99\n",
      "0.9639\n",
      "2000步所用的时间为398.977676s\n",
      "step 2100, entropy loss: 0.119193, l2_loss: 3949.894043, total loss: 0.395686\n",
      "0.98\n",
      "step 2200, entropy loss: 0.120854, l2_loss: 3949.551514, total loss: 0.397323\n",
      "0.98\n",
      "step 2300, entropy loss: 0.133598, l2_loss: 3949.179199, total loss: 0.410040\n",
      "0.97\n",
      "step 2400, entropy loss: 0.130940, l2_loss: 3948.815430, total loss: 0.407358\n",
      "0.98\n",
      "step 2500, entropy loss: 0.078484, l2_loss: 3948.432129, total loss: 0.354875\n",
      "1.0\n",
      "step 2600, entropy loss: 0.101803, l2_loss: 3948.067871, total loss: 0.378167\n",
      "0.99\n",
      "step 2700, entropy loss: 0.044290, l2_loss: 3947.688721, total loss: 0.320628\n",
      "1.0\n",
      "step 2800, entropy loss: 0.057234, l2_loss: 3947.305664, total loss: 0.333545\n",
      "1.0\n",
      "step 2900, entropy loss: 0.065127, l2_loss: 3946.909668, total loss: 0.341411\n",
      "0.99\n",
      "step 3000, entropy loss: 0.025370, l2_loss: 3946.542969, total loss: 0.301628\n",
      "1.0\n",
      "0.9772\n",
      "3000步所用的时间为586.750240s\n"
     ]
    }
   ],
   "source": [
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "start = time.time()\n",
    "for step in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    lr = 0.01\n",
    "    _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr})\n",
    "\n",
    "    if (step+1) % 100 == 0:\n",
    "        print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "           (step+1, loss, l2_loss_value, total_loss_value))\n",
    "        #Test trained model\n",
    "        correct_prediction = tf.equal(tf.argmax(net, 1), tf.argmax(y_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "    if (step+1) % 1000 == 0:\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels}))\n",
    "        end = time.time()\n",
    "        print(\"%d步所用的时间为%fs\"%(step+1,end-start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在卷积层手动使用不同的激活函数，训练6个epoch，其中relu最优，准确度达到0.9772，tanh次之，准确度为0.93，sigmoid最差，准确度为0.11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
